{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python3\n",
    "# -*- coding: utf-8 -*-\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "import argparse\n",
    "import random\n",
    "from tempfile import gettempdir\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'QuanSongCi.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "  with open(filename, 'r', encoding='utf-8') as f:\n",
    "    data = list(f.read())\n",
    "  return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 1903073\n"
     ]
    }
   ],
   "source": [
    "vocabulary = read_data(filename)\n",
    "print('Data size', len(vocabulary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary_size = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "  \"\"\"Process raw inputs into a dataset.\"\"\"\n",
    "  count = [['UNK', -1]]\n",
    "  count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    index = dictionary.get(word, 0)\n",
    "    if index == 0:  # dictionary['UNK']\n",
    "      unk_count += 1\n",
    "    data.append(index)\n",
    "  count[0][1] = unk_count\n",
    "  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "  return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,\n",
    "                                                            vocabulary_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dictionary.json','w',encoding='utf-8') as inf:\n",
    "    json.dump(dictionary,inf)\n",
    "    inf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('reverse_dictionary.json','w',encoding='utf-8') as inf:\n",
    "    json.dump(reverse_dictionary,inf)\n",
    "    inf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common words (+UNK) [['UNK', 1196], ('。', 149620), ('\\n', 117070), ('，', 108451), ('、', 19612)]\n",
      "Sample data [1503, 1828, 2, 2, 40, 613, 47, 9, 111, 117] ['潘', '阆', '\\n', '\\n', '酒', '泉', '子', '（', '十', '之']\n"
     ]
    }
   ],
   "source": [
    "del vocabulary  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_index = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "  global data_index\n",
    "  assert batch_size % num_skips == 0\n",
    "  assert num_skips <= 2 * skip_window\n",
    "  batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "  span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "  buffer = collections.deque(maxlen=span)\n",
    "  if data_index + span > len(data):\n",
    "    data_index = 0\n",
    "  buffer.extend(data[data_index:data_index + span])\n",
    "  data_index += span\n",
    "  for i in range(batch_size // num_skips):\n",
    "    context_words = [w for w in range(span) if w != skip_window]\n",
    "    words_to_use = random.sample(context_words, num_skips)\n",
    "    for j, context_word in enumerate(words_to_use):\n",
    "      batch[i * num_skips + j] = buffer[skip_window]\n",
    "      labels[i * num_skips + j, 0] = buffer[context_word]\n",
    "    if data_index == len(data):\n",
    "      #buffer[:] = data[:span]\n",
    "      buffer.extend(data[0:span])\n",
    "      \"\"\"\n",
    "      for word in data[:span]:\n",
    "        buffer.append(word)\n",
    "      \"\"\"\n",
    "      data_index = span\n",
    "    else:\n",
    "      buffer.append(data[data_index])\n",
    "      data_index += 1\n",
    "  # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "  data_index = (data_index + len(data) - span) % len(data)\n",
    "  return batch, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1828 阆 -> 1503 潘\n",
      "1828 阆 -> 2 \n",
      "\n",
      "2 \n",
      " -> 1828 阆\n",
      "2 \n",
      " -> 2 \n",
      "\n",
      "2 \n",
      " -> 40 酒\n",
      "2 \n",
      " -> 2 \n",
      "\n",
      "40 酒 -> 613 泉\n",
      "40 酒 -> 2 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "  print(batch[i], reverse_dictionary[batch[i]],\n",
    "        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1       # How many words to consider left and right.\n",
    "num_skips = 2         # How many times to reuse an input to generate a label.\n",
    "num_sampled = 64      # Number of negative examples to sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_size = 16     # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n",
    "\n",
    "learning_rate = 1.0\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "graph = tf.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-17-5bfc1a4b5baa>:38: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "  valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "  # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "  with tf.device('/cpu:0'):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "    # Construct the variables for the NCE loss\n",
    "    nce_weights = tf.Variable(\n",
    "        tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                            stddev=1.0 / math.sqrt(embedding_size)))\n",
    "    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "  # Compute the average NCE loss for the batch.\n",
    "  # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "  # time we evaluate the loss.\n",
    "  # Explanation of the meaning of NCE loss:\n",
    "  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/\n",
    "  loss = tf.reduce_mean(\n",
    "      tf.nn.nce_loss(weights=nce_weights,\n",
    "                     biases=nce_biases,\n",
    "                     labels=train_labels,\n",
    "                     inputs=embed,\n",
    "                     num_sampled=num_sampled,\n",
    "                     num_classes=vocabulary_size))\n",
    "\n",
    "  # Construct the SGD optimizer using a learning rate of 1.0.  \n",
    "  optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "  # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "  normalized_embeddings = embeddings / norm\n",
    "  valid_embeddings = tf.nn.embedding_lookup(\n",
    "      normalized_embeddings, valid_dataset)\n",
    "  similarity = tf.matmul(\n",
    "      valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "  # Add variable initializer.\n",
    "  init = tf.global_variables_initializer()\n",
    "\n",
    "  saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_steps = 500001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /home/fei/AI/csdn课程/第11周/作业/quiz-w10-code/ckpt/model-500000\n",
      "Average loss at step  0 :  4.239289283752441\n",
      "Nearest to 子: 牧, 跌, 乙, 乐, 姆, 潞, 荛, 闤,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 却, 总, 荏, 渐,\n",
      "Nearest to 醉: 笑, 睡, 醒, 酣, 困, 醺, 恣, 拚,\n",
      "Nearest to 云: 烟, 霞, 霭, 雾, 颅, 旆, 缥, 山,\n",
      "Nearest to 何: 多, 否, 邂, 昨, 还, 焉, 无, 那,\n",
      "Nearest to 未: 不, 乍, 初, 已, 易, 也, 难, 顿,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 对, 向, 想, 摘,\n",
      "Nearest to 天: 逍, 快, 帝, 云, 霓, 罨, 朗, 蠲,\n",
      "Nearest to 如: 似, 奈, 藁, 而, 漠, 鳏, 旱, 躇,\n",
      "Nearest to 水: 江, 山, 波, 颅, 溪, 睥, 澹, 泠,\n",
      "Nearest to 、: ，, 。, 怕, （, 尽, 酤, 恁, 苞,\n",
      "Nearest to 长: 永, 平, 当, 殇, 逢, 滚, 遥, 亶,\n",
      "Nearest to 歌: 唱, 舞, 箫, 遏, 曲, 乐, 阕, 硬,\n",
      "Nearest to 南: 北, 西, 东, 阡, 头, 熄, 鄞, 葛,\n",
      "Nearest to 酒: 屠, 盏, 杯, 蘋, 酿, 醺, 釂, 篘,\n",
      "Nearest to 此: 今, 兹, 每, 予, 良, 幸, 毋, 何,\n",
      "Average loss at step  2000 :  3.811271740913391\n",
      "Average loss at step  4000 :  3.859837305665016\n",
      "Average loss at step  6000 :  3.88313531768322\n",
      "Average loss at step  8000 :  3.827462136387825\n",
      "Average loss at step  10000 :  3.8308524883687496\n",
      "Nearest to 子: 牧, 跌, 鸪, 潞, 儿, 荛, 乙, 姆,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 渐, 总, 败, 釂,\n",
      "Nearest to 醉: 笑, 酣, 醒, 睡, 恣, 饮, 困, 须,\n",
      "Nearest to 云: 烟, 雾, 旆, 霞, 天, 杳, 缥, 霭,\n",
      "Nearest to 何: 岂, 否, 多, 邂, 无, 昨, 那, 焉,\n",
      "Nearest to 未: 不, 乍, 初, 已, 欲, 先, 也, 易,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 把, 摘, 与, 向,\n",
      "Nearest to 天: 云, 快, 空, 逍, 怡, 罨, 己, 朗,\n",
      "Nearest to 如: 似, 鳏, 漠, 藁, 旱, 奈, 间, 胥,\n",
      "Nearest to 水: 江, 山, 溪, 海, 颅, 碧, 睥, 蹲,\n",
      "Nearest to 、: ，, 。, （, ；, 閤, 酤, 似, 待,\n",
      "Nearest to 长: 永, 逢, 当, 平, 常, 殇, 自, 矰,\n",
      "Nearest to 歌: 唱, 箫, 乐, 曲, 遏, 舞, 酺, 阕,\n",
      "Nearest to 南: 北, 西, 东, 鄞, 熄, 阡, 头, 葛,\n",
      "Nearest to 酒: 屠, 醪, 酿, 釂, 盏, 杯, 醑, 柑,\n",
      "Nearest to 此: 每, 兹, 今, 毋, 良, 予, 幸, 朅,\n",
      "Average loss at step  12000 :  3.8720256943702696\n",
      "Average loss at step  14000 :  3.8640610896348955\n",
      "Average loss at step  16000 :  3.8941541635990142\n",
      "Average loss at step  18000 :  3.926900759935379\n",
      "Average loss at step  20000 :  3.90381880235672\n",
      "Nearest to 子: 牧, 跌, 乙, 潞, 乐, 荛, 姆, 闤,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 荏, 败, 渐,\n",
      "Nearest to 醉: 醒, 睡, 笑, 酣, 困, 羞, 醺, 饮,\n",
      "Nearest to 云: 烟, 旆, 颅, 雾, 缥, 髭, 杳, 山,\n",
      "Nearest to 何: 焉, 多, 邂, 底, 岂, 那, 昨, 否,\n",
      "Nearest to 未: 不, 乍, 已, 易, 初, 欲, 裛, 难,\n",
      "Nearest to 看: 觑, 趁, 见, 对, 听, 向, 与, 把,\n",
      "Nearest to 天: 逍, 快, 帝, 空, 庸, 蠲, 霓, 荏,\n",
      "Nearest to 如: 似, 漠, 藁, 奈, 鳏, 在, 践, 陀,\n",
      "Nearest to 水: 江, 山, 溪, 颅, 波, 睥, 涎, 蹲,\n",
      "Nearest to 、: ，, 。, 怕, （, 苑, 颤, 说, 共,\n",
      "Nearest to 长: 殇, 永, 平, 当, 逢, 健, 遥, 滚,\n",
      "Nearest to 歌: 唱, 曲, 舞, 乐, 遏, 侑, 郢, 阕,\n",
      "Nearest to 南: 北, 西, 东, 阡, 鄞, 头, 熄, 葛,\n",
      "Nearest to 酒: 屠, 醪, 酿, 盏, 醑, 杯, 斟, 釂,\n",
      "Nearest to 此: 今, 兹, 每, 幸, 予, 良, 毋, 朅,\n",
      "Average loss at step  22000 :  3.9410456166267394\n",
      "Average loss at step  24000 :  3.91437060880661\n",
      "Average loss at step  26000 :  3.92005450463295\n",
      "Average loss at step  28000 :  3.947467041373253\n",
      "Average loss at step  30000 :  3.917200621724129\n",
      "Nearest to 子: 牧, 跌, 潞, 乙, 乐, 儿, 鸪, 籍,\n",
      "Nearest to 尽: 了, 遍, 都, 总, 竟, 却, 荏, 渐,\n",
      "Nearest to 醉: 酣, 笑, 睡, 醒, 醺, 困, 饮, 须,\n",
      "Nearest to 云: 烟, 雾, 旆, 霞, 缥, 天, 颅, 髭,\n",
      "Nearest to 何: 岂, 焉, 邂, 多, 昨, 无, 那, 否,\n",
      "Nearest to 未: 不, 难, 乍, 易, 已, 初, 也, 顿,\n",
      "Nearest to 看: 见, 趁, 觑, 向, 听, 对, 摘, 把,\n",
      "Nearest to 天: 逍, 云, 快, 自, 山, 怡, 霓, 庸,\n",
      "Nearest to 如: 似, 漠, 藁, 奈, 鳏, 旱, 陀, 而,\n",
      "Nearest to 水: 江, 山, 睥, 泉, 派, 泠, 波, 蹲,\n",
      "Nearest to 、: ，, 。, （, 怕, 眨, ；, 恁, 楯,\n",
      "Nearest to 长: 永, 平, 当, 阊, 逢, 殇, 常, 正,\n",
      "Nearest to 歌: 唱, 乐, 舞, 箫, 遏, 阕, 曲, 郢,\n",
      "Nearest to 南: 北, 西, 阡, 东, 熄, 葛, 鄞, 头,\n",
      "Nearest to 酒: 屠, 醺, 酿, 蘋, 盏, 醪, 杯, 篘,\n",
      "Nearest to 此: 今, 兹, 每, 良, 幸, 旧, 毋, 年,\n",
      "Average loss at step  32000 :  3.83134565615654\n",
      "Average loss at step  34000 :  3.855050886750221\n",
      "Average loss at step  36000 :  3.8917055323123932\n",
      "Average loss at step  38000 :  3.831842257618904\n",
      "Average loss at step  40000 :  3.829099857389927\n",
      "Nearest to 子: 牧, 跌, 潞, 鸪, 乙, 煅, 荛, 缵,\n",
      "Nearest to 尽: 了, 遍, 都, 渐, 竟, 总, 共, 半,\n",
      "Nearest to 醉: 笑, 睡, 饮, 酣, 醒, 恣, 须, 困,\n",
      "Nearest to 云: 烟, 雾, 旆, 缥, 霞, 杳, 髭, 蓂,\n",
      "Nearest to 何: 岂, 多, 邂, 蒻, 无, 否, 昨, 不,\n",
      "Nearest to 未: 不, 乍, 已, 初, 也, 欲, 先, 易,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 把, 拂, 摘, 向,\n",
      "Nearest to 天: 空, 快, 逍, 霄, 云, 怡, 罨, 朗,\n",
      "Nearest to 如: 似, 鳏, 藁, 而, 旱, 陀, 奈, 漠,\n",
      "Nearest to 水: 江, 山, 溪, 纣, 颅, 派, 睥, 蹲,\n",
      "Nearest to 、: ，, 。, ；, 閤, （, 眨, 似, 惘,\n",
      "Nearest to 长: 永, 逢, 常, 平, 殇, 当, 修, 跎,\n",
      "Nearest to 歌: 唱, 乐, 遏, 箫, 舞, 曲, 侑, 咽,\n",
      "Nearest to 南: 北, 西, 东, 熄, 鄞, 阡, 楚, 头,\n",
      "Nearest to 酒: 屠, 酿, 醪, 盏, 釂, 醑, 杯, 斟,\n",
      "Nearest to 此: 每, 兹, 今, 予, 毋, 幸, 良, 一,\n",
      "Average loss at step  42000 :  3.8641249545812606\n",
      "Average loss at step  44000 :  3.8782363114356992\n",
      "Average loss at step  46000 :  3.9126245675086975\n",
      "Average loss at step  48000 :  3.922164429306984\n",
      "Average loss at step  50000 :  3.9089736213684083\n",
      "Nearest to 子: 牧, 跌, 乙, 乐, 荛, 潞, 姆, 缵,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 奈, 荏, 败,\n",
      "Nearest to 醉: 睡, 笑, 醒, 酣, 困, 醺, 饮, 须,\n",
      "Nearest to 云: 烟, 颅, 旆, 缥, 。, 杳, 髭, 霞,\n",
      "Nearest to 何: 多, 焉, 邂, 昨, 岂, 甚, 还, 那,\n",
      "Nearest to 未: 不, 乍, 已, 易, 欲, 难, 也, 初,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 对, 向, 与, 把,\n",
      "Nearest to 天: 逍, 快, 帝, 罨, 春, 空, 淝, 山,\n",
      "Nearest to 如: 似, 漠, 藁, 奈, 鳏, 在, 。, 于,\n",
      "Nearest to 水: 江, 山, 溪, 颅, 蹲, 波, 涎, 纣,\n",
      "Nearest to 、: ，, 。, 怕, （, 纻, ；, 作, 苑,\n",
      "Nearest to 长: 永, 殇, 平, 当, 遥, 逢, 牢, 常,\n",
      "Nearest to 歌: 唱, 舞, 曲, 遏, 箫, 乐, 阕, 郢,\n",
      "Nearest to 南: 北, 西, 阡, 东, 熄, 头, 鄞, 葛,\n",
      "Nearest to 酒: 屠, 酿, 醪, 盏, 醑, 杯, 斟, 蘋,\n",
      "Nearest to 此: 兹, 今, 每, 幸, 良, 予, 毋, 朅,\n",
      "Average loss at step  52000 :  3.9354915244579316\n",
      "Average loss at step  54000 :  3.9096644978523254\n",
      "Average loss at step  56000 :  3.913724987387657\n",
      "Average loss at step  58000 :  3.9318016543388365\n",
      "Average loss at step  60000 :  3.8898296933174135\n",
      "Nearest to 子: 牧, 跌, 潞, 乙, 乐, 儿, 籍, 鸪,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 共, 渐, 却,\n",
      "Nearest to 醉: 笑, 酣, 睡, 醒, 饮, 醺, 困, 恣,\n",
      "Nearest to 云: 旆, 雾, 烟, 天, 缥, 霞, 颅, 鸾,\n",
      "Nearest to 何: 岂, 多, 邂, 无, 焉, 昨, 否, 那,\n",
      "Nearest to 未: 不, 难, 乍, 已, 易, 顿, 欲, 也,\n",
      "Nearest to 看: 见, 觑, 趁, 向, 听, 摘, 把, 与,\n",
      "Nearest to 天: 云, 逍, 山, 快, 庸, 罨, 忡, 自,\n",
      "Nearest to 如: 似, 奈, 漠, 藁, 旱, 鳏, 而, 陀,\n",
      "Nearest to 水: 江, 山, 波, 派, 碧, 溪, 睥, 渺,\n",
      "Nearest to 、: ，, 。, （, 离, 怕, 觑, 逶, 楯,\n",
      "Nearest to 长: 永, 当, 平, 逢, 常, 殇, 阊, 亶,\n",
      "Nearest to 歌: 唱, 乐, 舞, 箫, 侑, 遏, 阕, 郢,\n",
      "Nearest to 南: 北, 西, 东, 阡, 熄, 葛, 鄞, 头,\n",
      "Nearest to 酒: 屠, 醺, 杯, 盏, 蘋, 酿, 釂, 醑,\n",
      "Nearest to 此: 今, 兹, 每, 幸, 良, 我, 毋, 旧,\n",
      "Average loss at step  62000 :  3.8501126133203507\n",
      "Average loss at step  64000 :  3.8607083430290223\n",
      "Average loss at step  66000 :  3.8682785514593125\n",
      "Average loss at step  68000 :  3.8299406161308287\n",
      "Average loss at step  70000 :  3.8379538592100144\n",
      "Nearest to 子: 牧, 跌, 潞, 鸪, 乙, 荛, 缵, 煅,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 渐, 总, 败, 半,\n",
      "Nearest to 醉: 笑, 酣, 醒, 睡, 饮, 恣, 困, 釂,\n",
      "Nearest to 云: 烟, 旆, 雾, 缥, 颅, 杳, 霞, 缔,\n",
      "Nearest to 何: 岂, 多, 邂, 蒻, 焉, 否, 无, 昨,\n",
      "Nearest to 未: 不, 已, 乍, 初, 欲, 先, 难, 易,\n",
      "Nearest to 看: 觑, 趁, 见, 摘, 听, 把, 与, 拂,\n",
      "Nearest to 天: 快, 空, 逍, 罨, 怡, 莹, 帝, 蠲,\n",
      "Nearest to 如: 似, 鳏, 藁, 漠, 奈, 旱, 胥, 依,\n",
      "Nearest to 水: 江, 山, 溪, 颅, 碧, 睥, 纣, 派,\n",
      "Nearest to 、: ，, 。, （, ；, 眨, 閤, 伴, 崆,\n",
      "Nearest to 长: 永, 常, 平, 逢, 殇, 修, 跎, 当,\n",
      "Nearest to 歌: 唱, 乐, 箫, 遏, 舞, 曲, 酺, 侑,\n",
      "Nearest to 南: 北, 西, 东, 熄, 鄞, 头, 阡, 楚,\n",
      "Nearest to 酒: 屠, 釂, 盏, 醪, 酿, 醑, 杯, 斟,\n",
      "Nearest to 此: 兹, 每, 今, 一, 幸, 我, 予, 毋,\n",
      "Average loss at step  72000 :  3.864754096508026\n",
      "Average loss at step  74000 :  3.873850431919098\n",
      "Average loss at step  76000 :  3.9311534914970396\n",
      "Average loss at step  78000 :  3.9090644994974135\n",
      "Average loss at step  80000 :  3.9196330350637436\n",
      "Nearest to 子: 牧, 跌, 乙, 乐, 荛, 潞, 姆, 鸪,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 共, 败, 荏,\n",
      "Nearest to 醉: 睡, 笑, 醒, 酣, 困, 醺, 羞, 须,\n",
      "Nearest to 云: 烟, 旆, 颅, 缥, 雾, 霞, 髭, 杳,\n",
      "Nearest to 何: 多, 焉, 邂, 岂, 甚, 那, 还, 昨,\n",
      "Nearest to 未: 不, 乍, 已, 欲, 初, 易, 也, 难,\n",
      "Nearest to 看: 觑, 见, 趁, 对, 听, 向, 与, 摘,\n",
      "Nearest to 天: 快, 逍, 空, 罨, 帝, 春, 蠲, 霓,\n",
      "Nearest to 如: 似, 藁, 漠, 奈, 鳏, 在, 旱, 佣,\n",
      "Nearest to 水: 江, 溪, 山, 颅, 蹲, 渚, 波, 涎,\n",
      "Nearest to 、: ，, 。, ；, 怕, （, 作, 忆, 想,\n",
      "Nearest to 长: 平, 当, 永, 殇, 逢, 牢, 萧, 滚,\n",
      "Nearest to 歌: 唱, 曲, 舞, 箫, 遏, 乐, 郢, 侑,\n",
      "Nearest to 南: 北, 西, 东, 阡, 鄞, 熄, 头, 葛,\n",
      "Nearest to 酒: 屠, 盏, 醪, 酿, 杯, 釂, 醑, 斟,\n",
      "Nearest to 此: 兹, 今, 每, 幸, 予, 良, 朅, 毋,\n",
      "Average loss at step  82000 :  3.93177588891983\n",
      "Average loss at step  84000 :  3.90358061003685\n",
      "Average loss at step  86000 :  3.9257741852998733\n",
      "Average loss at step  88000 :  3.9218505129814147\n",
      "Average loss at step  90000 :  3.8706736575365066\n",
      "Nearest to 子: 牧, 跌, 潞, 乙, 乐, 鸪, 籍, 荛,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 却, 奈, 渐,\n",
      "Nearest to 醉: 笑, 酣, 醒, 睡, 醺, 饮, 困, 宴,\n",
      "Nearest to 云: 天, 旆, 雾, 缥, 霞, 烟, 鸾, 颅,\n",
      "Nearest to 何: 岂, 多, 无, 焉, 邂, 否, 那, 昨,\n",
      "Nearest to 未: 不, 乍, 难, 欲, 易, 已, 初, 也,\n",
      "Nearest to 看: 见, 觑, 趁, 把, 对, 听, 向, 呼,\n",
      "Nearest to 天: 云, 逍, 快, 山, 罨, 空, 霓, 怡,\n",
      "Nearest to 如: 似, 漠, 藁, 鳏, 旱, 奈, 陀, 为,\n",
      "Nearest to 水: 山, 江, 波, 派, 睥, 溪, 渚, 泠,\n",
      "Nearest to 、: ，, 。, （, 恁, ；, 添, 楯, 轣,\n",
      "Nearest to 长: 永, 当, 平, 逢, 常, 正, 跎, 殇,\n",
      "Nearest to 歌: 唱, 乐, 舞, 箫, 遏, 郢, 侑, 曲,\n",
      "Nearest to 南: 北, 西, 东, 阡, 熄, 葛, 头, 鄞,\n",
      "Nearest to 酒: 屠, 盏, 酿, 杯, 釂, 醑, 醺, 篘,\n",
      "Nearest to 此: 兹, 今, 每, 幸, 年, 良, 何, 旧,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  92000 :  3.847808943748474\n",
      "Average loss at step  94000 :  3.877649039745331\n",
      "Average loss at step  96000 :  3.845300874590874\n",
      "Average loss at step  98000 :  3.8405739010572435\n",
      "Average loss at step  100000 :  3.8424102200865744\n",
      "Nearest to 子: 跌, 牧, 潞, 鸪, 荛, 缵, 乙, 姆,\n",
      "Nearest to 尽: 了, 遍, 都, 渐, 竟, 总, 半, 败,\n",
      "Nearest to 醉: 笑, 睡, 饮, 醒, 酣, 须, 恣, 拚,\n",
      "Nearest to 云: 雾, 烟, 缥, 旆, 霞, 颅, 杳, 缔,\n",
      "Nearest to 何: 无, 岂, 邂, 多, 否, 昨, 蒻, 不,\n",
      "Nearest to 未: 不, 已, 初, 乍, 欲, 先, 难, 才,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 摘, 与, 把, 拂,\n",
      "Nearest to 天: 空, 快, 罨, 逍, 蠲, 怡, 帝, 莹,\n",
      "Nearest to 如: 似, 鳏, 旱, 藁, 漠, 依, 胥, 而,\n",
      "Nearest to 水: 江, 山, 溪, 颅, 纣, 睥, 碧, 派,\n",
      "Nearest to 、: ，, 。, （, ；, 是, 惘, 奈, 怕,\n",
      "Nearest to 长: 永, 逢, 常, 殇, 平, 当, 修, 跎,\n",
      "Nearest to 歌: 唱, 遏, 乐, 箫, 舞, 咽, 曲, 侑,\n",
      "Nearest to 南: 北, 西, 东, 熄, 鄞, 阡, 头, 楚,\n",
      "Nearest to 酒: 屠, 盏, 酿, 釂, 醑, 醪, 杯, 斟,\n",
      "Nearest to 此: 兹, 每, 今, 一, 我, 良, 幸, 敧,\n",
      "Average loss at step  102000 :  3.8613646107912065\n",
      "Average loss at step  104000 :  3.884446521639824\n",
      "Average loss at step  106000 :  3.925555533885956\n",
      "Average loss at step  108000 :  3.900362438440323\n",
      "Average loss at step  110000 :  3.9356039980649946\n",
      "Nearest to 子: 牧, 跌, 乙, 乐, 姆, 潞, 鸪, 荛,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 奈, 荏, 却,\n",
      "Nearest to 醉: 醒, 睡, 笑, 酣, 困, 醺, 须, 羞,\n",
      "Nearest to 云: 烟, 颅, 旆, 雾, 缥, 髭, 纣, 杳,\n",
      "Nearest to 何: 邂, 焉, 岂, 多, 还, 那, 昨, 底,\n",
      "Nearest to 未: 不, 乍, 已, 易, 难, 初, 欲, 裛,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 对, 向, 照, 拂,\n",
      "Nearest to 天: 快, 逍, 帝, 自, 春, 蠲, 庸, 己,\n",
      "Nearest to 如: 似, 藁, 奈, 漠, 鳏, 旱, 鷁, 胥,\n",
      "Nearest to 水: 江, 山, 颅, 溪, 蹲, 涎, 睥, 纣,\n",
      "Nearest to 、: ，, 。, ；, （, 怕, 恐, 酤, 春,\n",
      "Nearest to 长: 当, 殇, 平, 永, 逢, 滚, 牢, 健,\n",
      "Nearest to 歌: 唱, 舞, 遏, 曲, 箫, 乐, 侑, 阕,\n",
      "Nearest to 南: 北, 西, 东, 阡, 鄞, 熄, 楚, 葛,\n",
      "Nearest to 酒: 屠, 盏, 醪, 酿, 杯, 釂, 醑, 斟,\n",
      "Nearest to 此: 兹, 今, 每, 予, 幸, 朅, 一, 毋,\n",
      "Average loss at step  112000 :  3.9172261838912963\n",
      "Average loss at step  114000 :  3.9085080811977386\n",
      "Average loss at step  116000 :  3.9158326733112334\n",
      "Average loss at step  118000 :  3.925838256716728\n",
      "Average loss at step  120000 :  3.8404624233245848\n",
      "Nearest to 子: 跌, 牧, 潞, 乙, 乐, 鸪, 儿, 姆,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 共, 奈, 渐,\n",
      "Nearest to 醉: 笑, 酣, 睡, 醒, 醺, 饮, 困, 宴,\n",
      "Nearest to 云: 旆, 天, 雾, 烟, 缥, 霞, 颅, 杳,\n",
      "Nearest to 何: 无, 多, 焉, 邂, 岂, 昨, 否, 甚,\n",
      "Nearest to 未: 不, 乍, 难, 易, 欲, 初, 已, 也,\n",
      "Nearest to 看: 觑, 见, 趁, 向, 对, 把, 听, 摘,\n",
      "Nearest to 天: 云, 逍, 快, 忡, 怡, 山, 自, 庸,\n",
      "Nearest to 如: 似, 藁, 鳏, 漠, 奈, 旱, 陀, 而,\n",
      "Nearest to 水: 江, 山, 波, 派, 溪, 睥, 海, 蹲,\n",
      "Nearest to 、: ，, 。, 恁, 眨, （, 酤, 添, 觑,\n",
      "Nearest to 长: 永, 当, 逢, 平, 常, 正, 殇, 跎,\n",
      "Nearest to 歌: 唱, 乐, 箫, 舞, 遏, 侑, 郢, 阕,\n",
      "Nearest to 南: 北, 西, 东, 阡, 熄, 鄞, 葛, 头,\n",
      "Nearest to 酒: 屠, 盏, 酿, 釂, 醪, 杯, 醑, 蘋,\n",
      "Nearest to 此: 今, 兹, 每, 幸, 年, 良, 旧, 有,\n",
      "Average loss at step  122000 :  3.8539118374586105\n",
      "Average loss at step  124000 :  3.883240090608597\n",
      "Average loss at step  126000 :  3.8323102279901504\n",
      "Average loss at step  128000 :  3.835338060259819\n",
      "Average loss at step  130000 :  3.8511934878230094\n",
      "Nearest to 子: 牧, 跌, 鸪, 潞, 姆, 荛, 缵, 乙,\n",
      "Nearest to 尽: 了, 遍, 都, 渐, 竟, 共, 总, 半,\n",
      "Nearest to 醉: 笑, 饮, 睡, 须, 醒, 酣, 宴, 困,\n",
      "Nearest to 云: 雾, 烟, 旆, 缥, 霞, 姗, 颅, 霭,\n",
      "Nearest to 何: 岂, 邂, 多, 焉, 那, 昨, 否, 无,\n",
      "Nearest to 未: 不, 初, 已, 乍, 欲, 先, 也, 易,\n",
      "Nearest to 看: 觑, 趁, 见, 听, 与, 把, 想, 观,\n",
      "Nearest to 天: 空, 快, 罨, 逍, 月, 莹, 云, 璀,\n",
      "Nearest to 如: 似, 鳏, 藁, 旱, 依, 漠, 胥, 而,\n",
      "Nearest to 水: 江, 山, 颅, 溪, 碧, 睥, 纣, 派,\n",
      "Nearest to 、: ，, 。, （, ；, 酤, 閤, 入, 冱,\n",
      "Nearest to 长: 永, 逢, 殇, 修, 常, 平, 当, 跎,\n",
      "Nearest to 歌: 唱, 乐, 箫, 遏, 舞, 咽, 曲, 侑,\n",
      "Nearest to 南: 北, 西, 东, 阡, 鄞, 熄, 头, 葛,\n",
      "Nearest to 酒: 屠, 釂, 盏, 杯, 醑, 醪, 酿, 篘,\n",
      "Nearest to 此: 兹, 每, 今, 一, 予, 敧, 良, 我,\n",
      "Average loss at step  132000 :  3.8501100583076475\n",
      "Average loss at step  134000 :  3.8836710159778596\n",
      "Average loss at step  136000 :  3.9279141327142715\n",
      "Average loss at step  138000 :  3.8931371124982834\n",
      "Average loss at step  140000 :  3.938911405920982\n",
      "Nearest to 子: 牧, 跌, 荛, 姆, 乙, 潞, 乐, 鸪,\n",
      "Nearest to 尽: 了, 遍, 都, 总, 竟, 奈, 荏, 败,\n",
      "Nearest to 醉: 醒, 睡, 酣, 笑, 醺, 困, 饮, 宴,\n",
      "Nearest to 云: 烟, 颅, 缥, 旆, 髭, 山, 霞, 汛,\n",
      "Nearest to 何: 焉, 邂, 多, 岂, 那, 还, 今, 昨,\n",
      "Nearest to 未: 不, 乍, 已, 易, 欲, 难, 顿, 也,\n",
      "Nearest to 看: 觑, 趁, 听, 对, 见, 向, 与, 任,\n",
      "Nearest to 天: 逍, 快, 帝, 蠲, 空, 霓, 莹, 云,\n",
      "Nearest to 如: 似, 奈, 藁, 漠, 鳏, 旱, 佣, 鷁,\n",
      "Nearest to 水: 江, 山, 颅, 溪, 涎, 蹲, 波, 派,\n",
      "Nearest to 、: ，, 。, ；, 怕, （, 恐, 得, 苞,\n",
      "Nearest to 长: 平, 逢, 当, 永, 殇, 滚, 牢, 修,\n",
      "Nearest to 歌: 唱, 舞, 乐, 曲, 遏, 箫, 侑, 酺,\n",
      "Nearest to 南: 北, 西, 东, 阡, 鄞, 熄, 楚, 头,\n",
      "Nearest to 酒: 屠, 醪, 盏, 酿, 醑, 杯, 蘋, 釂,\n",
      "Nearest to 此: 兹, 今, 每, 予, 幸, 朅, 良, 我,\n",
      "Average loss at step  142000 :  3.9235538518428803\n",
      "Average loss at step  144000 :  3.904041269659996\n",
      "Average loss at step  146000 :  3.919531107187271\n",
      "Average loss at step  148000 :  3.917898445367813\n",
      "Average loss at step  150000 :  3.8253496439456938\n",
      "Nearest to 子: 跌, 牧, 潞, 儿, 鸪, 乐, 乙, 荛,\n",
      "Nearest to 尽: 了, 遍, 都, 竟, 总, 共, 渐, 奈,\n",
      "Nearest to 醉: 笑, 酣, 睡, 醺, 醒, 饮, 困, 酒,\n",
      "Nearest to 云: 旆, 雾, 缥, 烟, 天, 霞, 颅, 鸾,\n",
      "Nearest to 何: 无, 邂, 多, 岂, 焉, 那, 否, 昨,\n",
      "Nearest to 未: 不, 难, 乍, 已, 易, 欲, 顿, 也,\n",
      "Nearest to 看: 觑, 见, 趁, 向, 听, 把, 对, 摘,\n",
      "Nearest to 天: 逍, 云, 快, 罨, 空, 忡, 怡, 自,\n",
      "Nearest to 如: 似, 藁, 奈, 鳏, 而, 成, 漠, 旱,\n",
      "Nearest to 水: 江, 山, 波, 派, 泉, 溪, 睥, 泠,\n",
      "Nearest to 、: ，, 。, 添, 眨, 恁, （, 愁, 觑,\n",
      "Nearest to 长: 永, 平, 逢, 当, 滚, 正, 遥, 常,\n",
      "Nearest to 歌: 唱, 乐, 箫, 舞, 遏, 郢, 曲, 侑,\n",
      "Nearest to 南: 西, 北, 东, 阡, 熄, 葛, 鄞, 湖,\n",
      "Nearest to 酒: 盏, 屠, 酿, 醺, 杯, 醑, 醪, 篘,\n",
      "Nearest to 此: 今, 兹, 每, 良, 旧, 幸, 何, 年,\n",
      "Average loss at step  152000 :  3.8586636370420457\n",
      "Average loss at step  154000 :  3.8840362901687624\n",
      "Average loss at step  156000 :  3.8314248073101043\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "  ckpt_path = tf.train.latest_checkpoint('./ckpt/')\n",
    "  # We must initialize all variables before we use them.\n",
    "  if ckpt_path:\n",
    "    saver.restore(session, ckpt_path)\n",
    "  else:\n",
    "    init.run()\n",
    "    print('Initialized')\n",
    "\n",
    "  average_loss = 0\n",
    "  for step in xrange(num_steps):\n",
    "    batch_inputs, batch_labels = generate_batch(\n",
    "        batch_size, num_skips, skip_window)\n",
    "    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "    # We perform one update step by evaluating the optimizer op (including it\n",
    "    # in the list of returned values for session.run()\n",
    "    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "    average_loss += loss_val\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "      if step > 0:\n",
    "        average_loss /= 2000\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print('Average loss at step ', step, ': ', average_loss)\n",
    "      average_loss = 0\n",
    "\n",
    "    # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "    if step % 10000 == 0:\n",
    "      sim = similarity.eval()\n",
    "      for i in xrange(valid_size):\n",
    "        valid_word = reverse_dictionary[valid_examples[i]]\n",
    "        top_k = 8  # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "        log_str = 'Nearest to %s:' % valid_word\n",
    "        for k in xrange(top_k):\n",
    "          close_word = reverse_dictionary[nearest[k]]\n",
    "          log_str = '%s %s,' % (log_str, close_word)\n",
    "        print(log_str)\n",
    "  final_embeddings = normalized_embeddings.eval()\n",
    "\n",
    "  saver.save(session, os.path.join(os.getcwd(), 'ckpt/model'), global_step=step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('embedding.npy', final_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pylint: disable=missing-docstring\n",
    "# Function to draw visualization of distance between embeddings.\n",
    "def plot_with_labels(low_dim_embs, labels, filename):\n",
    "  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'\n",
    "  plt.figure(figsize=(18, 18))  # in inches\n",
    "  for i, label in enumerate(labels):\n",
    "    x, y = low_dim_embs[i, :]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label,\n",
    "                 xy=(x, y),\n",
    "                 xytext=(5, 2),\n",
    "                 textcoords='offset points',\n",
    "                 ha='right',\n",
    "                 va='bottom')\n",
    "\n",
    "  print('saving: '+filename)\n",
    "  plt.savefig(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "  # pylint: disable=g-import-not-at-top\n",
    "  from sklearn.manifold import TSNE\n",
    "  import matplotlib.pyplot as plt\n",
    "  from pylab import mpl\n",
    "  mpl.rcParams['font.sans-serif'] = ['Vera']\n",
    "  mpl.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "  tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')\n",
    "  plot_only = 500\n",
    "  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])\n",
    "  labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n",
    "  plot_with_labels(low_dim_embs, labels, os.path.join(os.getcwd(), 'tsne.png'))\n",
    "\n",
    "except ImportError as ex:\n",
    "  print('Please install sklearn, matplotlib, and scipy to show embeddings.')\n",
    "  print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "matplotlib.matplotlib_fname()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
